/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __LINUX_SWIOTLB_H
#define __LINUX_SWIOTLB_H

#include <linux/device.h>
#include <linux/dma-direction.h>
#include <linux/init.h>
#include <linux/types.h>
#include <linux/limits.h>
#include <linux/spinlock.h>
#include <linux/workqueue.h>

struct device;
struct page;
struct scatterlist;

enum swiotlb_force {
	SWIOTLB_NORMAL,		/* Default - depending on HW DMA mask etc. */
	SWIOTLB_FORCE,		/* swiotlb=force */
	SWIOTLB_NO_FORCE,	/* swiotlb=noforce */
};

/*
 * Maximum allowable number of contiguous slabs to map,
 * must be a power of 2.  What is the appropriate value ?
 * The complexity of {map,unmap}_single is linearly dependent on this value.
 */
#define IO_TLB_SEGSIZE	128

/*
 * log of the size of each IO TLB slab.  The number of slabs is command line
 * controllable.
 */
#define IO_TLB_SHIFT 11
#define IO_TLB_SIZE (1 << IO_TLB_SHIFT)

extern void swiotlb_init(int verbose);
int swiotlb_init_with_tbl(char *tlb, unsigned long nslabs, int verbose);
extern unsigned long swiotlb_nr_tbl(void);
unsigned long swiotlb_size_or_default(void);
extern int swiotlb_late_init_with_tbl(char *tlb, unsigned long nslabs);
extern int swiotlb_late_init_with_default_size(size_t default_size);
extern void __init swiotlb_update_mem_attributes(void);

/*
 * Enumeration for sync targets
 */
enum dma_sync_target {
	SYNC_FOR_CPU = 0,
	SYNC_FOR_DEVICE = 1,
};

phys_addr_t swiotlb_tbl_map_single(struct device *hwdev, phys_addr_t phys,
		size_t mapping_size, size_t alloc_size,
		enum dma_data_direction dir, unsigned long attrs);

extern void swiotlb_tbl_unmap_single(struct device *hwdev,
				     phys_addr_t tlb_addr,
				     size_t mapping_size,
				     size_t alloc_size,
				     enum dma_data_direction dir,
				     unsigned long attrs);

extern void swiotlb_tbl_sync_single(struct device *hwdev,
				    phys_addr_t tlb_addr,
				    size_t size, enum dma_data_direction dir,
				    enum dma_sync_target target);

dma_addr_t swiotlb_map(struct device *dev, phys_addr_t phys,
		size_t size, enum dma_data_direction dir, unsigned long attrs);

#ifdef CONFIG_SWIOTLB
extern enum swiotlb_force swiotlb_force;

/**
 * struct dev_dma_io_tlb - The wrapper for device dma io tlb structure
 * @dma_io_tlb_pools:	List of transient swiotlb memory pools.
 * @dma_io_tlb_lock:	Protects changes to the list of active pools.
 * @dma_uses_io_tlb: %true if device has used the software IO TLB.
 */
struct dev_dma_io_tlb {
	struct list_head dma_io_tlb_pools;
	spinlock_t dma_io_tlb_lock;
	bool dma_uses_io_tlb;
};

/**
 * struct io_tlb_pool - IO TLB memory pool descriptor
 * @start:	The start address of the swiotlb memory pool. Used to do a quick
 *		range check to see if the memory was in fact allocated by this
 *		API.
 * @end:	The end address of the swiotlb memory pool. Used to do a quick
 *		range check to see if the memory was in fact allocated by this
 *		API.
 * @nslabs:	The number of IO TLB slots between @start and @end. For the
 *		default swiotlb, this can be adjusted with a boot parameter,
 *		see setup_io_tlb_npages().
 * @used:	The number of used IO TLB block.
 * @list:	The free list describing the number of free entries available
 *		from each index.
 * @orig_addr:	The original address corresponding to a mapped entry.
 * @late_alloc:	%true if allocated using the page allocator
 * @nareas:	Number of areas in the pool.
 * @area_nslabs: Number of slots in each area.
 * @areas:	Array of memory area descriptors.
 * @node:	Member of the IO TLB memory pool list.
 * @rcu:	RCU head for swiotlb_dyn_free().
 * @transient:  %true if transient memory pool.
 */
struct io_tlb_pool {
	phys_addr_t start;
	phys_addr_t end;
	unsigned long nslabs;
	unsigned long used;
	unsigned int *list;
	phys_addr_t *orig_addr;
	bool late_alloc;
	unsigned int nareas;
	unsigned int area_nslabs;
	struct io_tlb_area *areas;
#ifdef CONFIG_SWIOTLB_DYNAMIC
	struct list_head node;
	struct rcu_head rcu;
	bool transient;
#endif
};

/**
 * struct io_tlb_mem - Software IO TLB allocator
 * @defpool:	Default (initial) IO TLB memory pool descriptor.
 * @nslabs:	Total number of IO TLB slabs in all pools.
 * @debugfs:	The dentry to debugfs.
 * @can_grow:	%true if more pools can be allocated dynamically.
 * @phys_limit:	Maximum allowed physical address.
 * @lock:	Lock to synchronize changes to the list.
 * @pools:	List of IO TLB memory pool descriptors (if dynamic).
 * @dyn_alloc:	Dynamic IO TLB pool allocation work.
 * @transient_nslabs:	The total number of slots in all transient pools that
 *			are currently used across all areas.
 */
struct io_tlb_mem {
	struct io_tlb_pool defpool;
	unsigned long nslabs;
	struct dentry *debugfs;
#ifdef CONFIG_SWIOTLB_DYNAMIC
	bool can_grow;
	u64 phys_limit;
	spinlock_t lock;
	struct list_head pools;
	struct work_struct dyn_alloc;
	atomic_long_t transient_nslabs;
#endif
};
extern struct io_tlb_mem io_tlb_default_mem;

#ifdef CONFIG_SWIOTLB_DYNAMIC

struct io_tlb_pool *swiotlb_find_pool(struct device *dev, phys_addr_t paddr);

#else

static inline struct io_tlb_pool *swiotlb_find_pool(struct device *dev,
						    phys_addr_t paddr)
{
	return &io_tlb_default_mem.defpool;
}

#endif

static inline bool is_swiotlb_buffer(struct device *dev, phys_addr_t paddr)
{
#ifdef CONFIG_SWIOTLB_DYNAMIC
	/*
	 * All SWIOTLB buffer addresses must have been returned by
	 * swiotlb_tbl_map_single() and passed to a device driver.
	 * If a SWIOTLB address is checked on another CPU, then it was
	 * presumably loaded by the device driver from an unspecified private
	 * data structure. Make sure that this load is ordered before reading
	 * dma_uses_io_tlb here and mem->pools in swiotlb_find_pool().
	 *
	 * This barrier pairs with smp_mb() in swiotlb_find_slots().
	 */
	smp_rmb();
	return READ_ONCE(dev->iotlb->dma_uses_io_tlb) &&
		swiotlb_find_pool(dev, paddr);
#else
	struct io_tlb_mem *mem = &io_tlb_default_mem;

	return paddr >= mem->defpool.start && paddr < mem->defpool.end;
#endif
}

void __init swiotlb_exit(void);
unsigned int swiotlb_max_segment(void);
size_t swiotlb_max_mapping_size(struct device *dev);
bool is_swiotlb_active(void);
#else
#define swiotlb_force SWIOTLB_NO_FORCE
static inline bool is_swiotlb_buffer(struct device *dev, phys_addr_t paddr)
{
	return false;
}
static inline void swiotlb_exit(void)
{
}
static inline unsigned int swiotlb_max_segment(void)
{
	return 0;
}
static inline size_t swiotlb_max_mapping_size(struct device *dev)
{
	return SIZE_MAX;
}

static inline bool is_swiotlb_active(void)
{
	return false;
}
#endif /* CONFIG_SWIOTLB */

#ifdef CONFIG_SWIOTLB_DYNAMIC
int swiotlb_dev_init(struct device *dev);
void swiotlb_dev_remove(struct device *dev);
#else
static inline int swiotlb_dev_init(struct device *dev)
{
	return 0;
}
static inline void swiotlb_dev_remove(struct device *dev)
{
}
#endif

extern void swiotlb_print_info(void);
extern void swiotlb_set_max_segment(unsigned int);

#endif /* __LINUX_SWIOTLB_H */
